#include <fstream>
#include <iostream>
#include <unistd.h>
#include <unordered_set>
#include <regex>
#include <cmath>
#include <utility>
#include <algorithm> // for std::copy_if
#include "client/create_weblib/PageLibPreprocessor.h"
#include "client/create_weblib/WebPage.h"
#include "client/Configuration.h"
#include "tinyxml/tinyxml2.h"
#include "cppjieba/Jieba.hpp"
#include "simhash/Simhasher.hpp"



using std::ifstream;
using std::ofstream;
using std::ostringstream;
using std::unordered_set;
using std::cout;
using std::cerr;
using std::log2;
using tinyxml2::XMLDocument;
using tinyxml2::XMLError;
using tinyxml2::XML_SUCCESS;
using tinyxml2::XMLElement;

PageLibPreprocessor::PageLibPreprocessor()
{
}

void PageLibPreprocessor::doProcess(){
    readInfoFromFile();
    //cutRedundantPage();
    buildInvertindexMap();
}

void PageLibPreprocessor::readInfoFromFile(){
    cout << "strat read info file" << '\n';
    string list_path = getcwd(NULL, 0);
    list_path += "/data/pagelist.dat";

    if (!ifstream(list_path).is_open())
        return;
    set<string> web = Configuration::getInstance(getcwd(NULL, 0))->find_config("web_yuliao");
    unordered_set<uint64_t> hash_lib;
    splitTool* split_tool = new splitToolCppJieba;

    cout << "getting wash word" << '\n';
    string word_s;
    set<string> stop_words;
    ifstream ifs_stop;
    ifs_stop.open(*((Configuration::getInstance(getcwd(NULL, 0))->find_config("stop_words_cn")).begin()));
    while (ifs_stop >> word_s)
        stop_words.insert(word_s);
    ifs_stop.open(*((Configuration::getInstance(getcwd(NULL, 0))->find_config("stop_words_en")).begin()));
    while (ifs_stop >> word_s)
        stop_words.insert(word_s);

    cout << "getting xml" << '\n';
    int docId = 1;
    for(auto &i : web){
        // 创建一个 TinyXML 文档对象
        XMLDocument doc;

        // 加载 XML 文件
        XMLError result = doc.LoadFile(i.c_str());
        if (result != XML_SUCCESS)
        {
            cerr << "Failed to load file: " << doc.ErrorIDToName(result) << '\n';
            return;
        }

        // 获取根元素
        XMLElement *channel = doc.FirstChildElement("rss")->FirstChildElement("channel");
        if (channel == nullptr)
        {
            cerr << "No channel element found!" << '\n';
            return;
        }

        for (XMLElement *item = channel->FirstChildElement("item"); item != nullptr; item = item->NextSiblingElement("item"))
        {
            const char *itemTitle = item->FirstChildElement("title")->GetText();
            const char *itemLink = item->FirstChildElement("link")->GetText();
            const char *itemDescription = item->FirstChildElement("description")->GetText();

            if (!itemTitle || !itemLink || !itemDescription)
            {
                if(!itemTitle)
                    cout << "跳过缺失信息的项" << '\n';
                if (!itemLink)
                    cout << "跳过缺失信息的项" << '\n';
                if (!itemDescription)
                    cout << "跳过缺失信息的项" << '\n';
                continue;
            }

            string doc; 
            ostringstream oss;
            oss << "<doc>\n\t<docid>" << docId << "</docid>\n\t" 
                << "<title>" << itemTitle << "</title>\n\t"
                << "<link>" << itemLink << "</link>\n\t"
                << "<content>" << itemDescription << "</content>\n" << "</doc>";

            doc = oss.str();

            vector<string> key_words;
            split_tool->cut(doc, key_words);

            vector<string> key_words_clean;
            for(auto &i : key_words){
                if(stop_words.find(i) == stop_words.end()){
                    key_words_clean.push_back(i);
                }
            }

            simhash::Simhasher simhasher("resource/dict/jieba.dict.utf8", "resource/dict/hmm_model.utf8", "resource/dict/idf.utf8", "resource/dict/stop_words.utf8");
            size_t topN = 5;
            uint64_t u64 = 0;
            vector<pair<string, double>> res;
            simhasher.extract(doc, res, topN);
            simhasher.make(doc, topN, u64);
            cout << "文本：\"" << doc << "\"" << '\n';
            cout << "关键词序列是: " << res << '\n';
            cout << "simhash值是: " << u64 << '\n';

            if(hash_lib.find(u64) != hash_lib.end())
                continue;

            WebPage page("", *Configuration::getInstance(getcwd(NULL, 0)));
            page._doc = doc;
            page._docId = docId;
            page._docTitle = itemTitle;
            page._docUrl = itemLink;
            page._docContent = itemDescription;

            _pageList.push_back(page);

            string pagelist_path = (getcwd(NULL, 0));
            pagelist_path += "/data/pagelist.dat";
            ofstream ofs;

            ofs.open(pagelist_path, std::ios::app);
            ofs << doc << '\n';

            _offsetLib[docId] = {6 * (docId - 1) + 1, 6 * (docId)};

            string offset_list_path = getcwd(NULL, 0);
            offset_list_path += "/data/offset_lib.dat";
            ofstream outfile(offset_list_path, std::ios::app);

            outfile << docId << ' ' << _offsetLib[docId].first << ' ' << _offsetLib[docId].second << '\n';
            outfile.close();

            for(auto &i: key_words_clean){
                if (_word_frequnce.find(i) == _word_frequnce.end())
                {
                    if (_word_frequnce[i].find(docId) != _word_frequnce[i].end()){
                        _word_frequnce[i][docId]++;
                    }else{
                        _word_frequnce[i].insert({docId, 1});
                    }
                }else{
                    _word_frequnce[i].insert({docId, 1});
                }
            }

            docId++;
        }
    }

    return;
}

void PageLibPreprocessor::cutRedundantPage(){}

void PageLibPreprocessor::buildInvertindexMap(){
    cout << "statisticing pow" << '\n';
    unordered_map<int, vector<double>> stiatistic;
    double N = _pageList.size();
    for(auto &i: _word_frequnce){
        double DF = i.second.size();
        for(auto &j: i.second){
            double TF = j.second;
            double W = TF * (log2(N / (DF + 1) + 1));
            _invertIndexTable[i.first].push_back({j.first, W});
        }
    }
    cout << "statisticing complete" << '\n';

    cout << "calculating" << '\n';
    int id = 1;
    double main_ponit;
    vector<double> totle;
    totle.reserve(N);
    for(;id<=N;id++){
        main_ponit = 0;
        for (auto &i : _invertIndexTable)
        {
            for (const auto &p : i.second)
            {
                if (p.first == id){
                    main_ponit += pow(p.second, 2);
                    break;
                } 
            }
        }
        totle.push_back(sqrt(main_ponit));
    }

    for(auto &i : _invertIndexTable){
        for(auto &j: i.second){
            j.second /= totle[(j.first - 1)];
        }
    }

    // for(auto &i: _invertIndexTable){
    //     for(auto &j: i.second){
    //         cout << i.first << ' ';
    //         cout << j.first << ' ' << j.second << '\n';
    //     }
    // }

    // 存储倒排索引到文件
    string path = getcwd(NULL, 0);
    path += "/data/invert_index.dat";
    ofstream outfile(path);
    for (const auto &entry : _invertIndexTable)
    {
        for (const auto &loc : entry.second)
        {
            outfile << entry.first << " " << loc.first << " " << loc.second << '\n';
        }
        outfile << '\n';
    }
    outfile.close();
}

void PageLibPreprocessor::storeOnDisk(){}